package com.lagou.hadoop.sort;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * 排序执行driver类
 * @Author chenkl
 * @Date 2020/8/9 17:43
 */
public class SortDriver {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        //1. 获取job
        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration,"sortDriver");

        //2. 设置job 执行的driver,mapper,reducer类
        job.setJarByClass(SortDriver.class);
        job.setMapperClass(SortMapper.class);
        job.setReducerClass(SortReducer.class);
        //3. 设置Map输出参数类型
        job.setMapOutputKeyClass(SortBean.class);
        job.setMapOutputValueClass(NullWritable.class);
        //4. 设置最终输出参数类型
        job.setOutputKeyClass(SortBean.class);
        job.setOutputValueClass(NullWritable.class);

        //4.1 全排序，必须设置一个reduceTask,默认就是一个,也可以直接进行设置
        job.setNumReduceTasks(1);
        //4.2 有多个小文件，可以对这些小文件进行合并处理，文件最大不到1M，设置缓存为1M，应该只会生成一个分片，即一个MapTask处理
        job.setInputFormatClass(CombineTextInputFormat.class);
        CombineTextInputFormat.setMaxInputSplitSize(job,1048576);
        //5. 设置输入文件目录
        FileInputFormat.setInputPaths(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));
        //6. 执行job
        boolean flag = job.waitForCompletion(true);
        System.exit(flag?0:1);
    }

}
